#!/usr/bin/perl -w
use DBI;
use warnings;
use Encode;
use utf8;
binmode(STDIN,':encoding(utf8)');
binmode(STDOUT,':encoding(utf8)');
binmode(STDERR,':encoding(utf8)');  
use open ":encoding(utf8)";
use LWP::Protocol::https;
use LWP::UserAgent;
use Digest::MD5 qw(md5 md5_hex md5_base64);
use Time::Local;
use Date::Parse;
#use strict;

&insert_news(1);

sub insert_news(){

    my $page= $_[0];
	#my $flag=0;
    #my $count;
	my $driver="DBI:mysql";
	my $database="news";
	my $host="localhost";
	my $user="root";
	my $password="192510";
	my $db=DBI->connect("$driver:database=$database;host=$host;user=$user;password=$password");
	

	#得到前几天的时间
	my $current_time=$db->prepare("select UNIX_TIMESTAMP(date_sub(curdate(),interval 7 day));");
	$current_time->execute();
	my @current_time2=$current_time->fetchrow_array;
	my $current_time3=$current_time2[0];
	
	my $url="https://www.seebug.org/vuldb/vulnerabilities?page=$page";
	my ($ua, $request , $response , $content);
	$ua = LWP::UserAgent->new(ssl_opts => { verify_hostname => 0 });
	$request = HTTP::Request->new('GET',$url);
	$response = $ua->request($request);
	
	
	if($response->code ne "200"){

		}else{
			$content = $response->content;
			my $html=decode("utf-8",$content);
			my @array=split(/<td class="vul-title-wrapper"><a class="vul-title" title="/,$html);
			my $array_num=scalar @array;
			#print $array_num;
			for(my $i=1;$i<$array_num;$i++){
				#link
				my $href=$array[$i-1];
				my @href2=split(/<td><a href="/,$href);
				my $href3=$href2[1];
				my @href4=split(/">/,$href3);
				my $href5=$href4[0];
				my $rel_href="https://www.seebug.org".$href5;
				#print "$rel_href\n";
				
			 	#时间
				my $ptime=$array[$i];
				my @ptime2=split(/<\/td>/,$href4[2]);
				my $ptime3=$ptime2[0];
				#print "$ptime3\n";
				
				# #title
				my $title=$array[$i];
				my @title3=split(/href=/,$title);
				my $rel_title=$title3[0];
				$rel_title=~s/"//g;
				#print "$rel_title\n";

				my ($ua, $request , $response , $content2);
				$ua = LWP::UserAgent->new(ssl_opts => { verify_hostname => 0 });
				$request = HTTP::Request->new('GET',$rel_href);
				$response = $ua->request($request);
				$content2 = $response->content;
				my $html2=decode("utf-8",$content2);
				
				#漏洞类型
				my @loudong1=split(/<dd><a href="\/category\//,$html2);			
				my @loudong2=split(/">/,$loudong1[1]);
				my $loudong4=$loudong2[0];
				#print "$loudong4\n";

				#漏洞作者

				# my @author1=split(/<dd>/,$html2);
				# my @author2=split(/<\/dd>/,$author1[9]);
				# my @author3=split(/<a.*>/,$author2[0]);
				# my @author5=split(/<\/a>/,$author3[1]);
				# my $author4=$author5[0];
				# $author4=~s/\s//g;
				#print "$author4\n";

				my @author1=split(/<dd>/,$html2);
				my @author2=split(/<\/dd>/,$author1[9]);
				my @author3=split(/<a.*>/,$author2[0]);
				my $rel_author=$author3[0];
				$rel_author=~s/\s//g;

				my $web=7;
				my $m=$rel_title.$loudong4;
				Encode::_utf8_off($m);
				my $md5=md5($m);

				if($rel_author eq "匿名"){
					my $author4=$rel_author;
					$author4=~s/\s//g;
					#print "$author4\n";
					
					
					my $bj_publishtime=str2time("$ptime3 GMT");
					if($bj_publishtime>=$current_time3){
						my $q_title=$rel_title;
						$q_title=~s/\s//g;

						my $q_href=$rel_href;
						$q_href=~s/\s//g;

						my $q_author=$author4;
						$q_author=~s/\s//g;


						if($q_title ne "" and $q_href ne "" and $q_author ne ""){
							my $sql = "SET NAMES UTF8";
							my $sth=$db->prepare($sql);
							$sth=$db->prepare(q{insert ignore into news(title,web,author,publishtime,content,link,nmd5)values(?,?,?,?,?,?,?)})or die $db->errstr;
							$sth->execute($rel_title,$web,$author4,$ptime3,$loudong4,$rel_href,$md5)or die $db->errstr;
							print "$author4\n";
						}
					}else{
						return;
					}
				}else{
					my @author5=split(/<\/a>/,$author3[1]);
					my $author4=$author5[0];
					$author4=~s/\s//g;
					#print "$author4\n";
					
					
					my $bj_publishtime=str2time("$ptime3 GMT");
					if($bj_publishtime>=$current_time3){
						my $q_title=$rel_title;
						$q_title=~s/\s//g;

						my $q_href=$rel_href;
						$q_href=~s/\s//g;

						my $q_author=$author4;
						$q_author=~s/\s//g;
						if($q_title ne "" and $q_href ne "" and $q_author ne ""){
							my $sql = "SET NAMES UTF8";
							my $sth=$db->prepare($sql);
							$sth=$db->prepare(q{insert ignore into news(title,web,author,publishtime,content,link,nmd5)values(?,?,?,?,?,?,?)})or die $db->errstr;
							$sth->execute($rel_title,$web,$author4,$ptime3,$loudong4,$rel_href,$md5)or die $db->errstr;
							print "$author4\n";
						}
					}else{
						return;
					}
				}
		
			}
			sleep(1);
			$page=$page+1;
			&insert_news($page);
		}
		$current_time->finish;
		$db->disconnect();
	

}

sub randomagent
{
## FireFox
# Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0
# Mozilla/5.0 (Windows NT 6.1; WOW64; rv:13.0) Gecko/20100101 Firefox/13.0.1
# Mozilla/5.0 (Windows NT 6.1; WOW64; rv:14.0) Gecko/20100101 Firefox/14.0.1
# Chrome
# Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.168 Safari/535.19 QIHU 360EE
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1
##Safari
# Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13
##Sogou
# Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0
##Baidu
# Mozilla/5.0 (Windows; U; Windows NT 6.1; zh_CN) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/18.0 BIDUBrowser/2.6 Safari/534.7
##QQ
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.7 (KHTML, like Gecko) Chrome/20.0.1099.0 Safari/536.7 QQBrowser/6.14.15138.201
##Maxthon
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.12 (KHTML, like Gecko) Maxthon/3.4.2.3000 Chrome/18.0.966.0 Safari/535.12
##LBBROWSER
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER
##TheWorld
# Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.79 Safari/535.11 QIHU THEWORLD
	my @agent_conf=('Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0',
	'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:13.0) Gecko/20100101 Firefox/13.0.1',
	'Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.168 Safari/535.19 QIHU 360EE',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1',
	'Mozilla/5.0 (Windows; U; Windows NT 5.2) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13',
	'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
	'Mozilla/5.0 (Windows; U; Windows NT 6.1; zh_CN) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/18.0 BIDUBrowser/2.6 Safari/534.7',
	'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.12 (KHTML, like Gecko) Maxthon/3.4.2.3000 Chrome/18.0.966.0 Safari/535.12',
	'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER',
	'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.79 Safari/535.11 QIHU THEWORLD');
	my $int=int(rand(12));
	return $agent_conf[$int];
}
